import scrapy
import re
from solidot.items import NewsItem



class QuotesSpider(scrapy.Spider):

    name='solidot'
    allowed_domains=['solidot.org']
    start_urls = ['https://www.solidot.org',
                  'http://www.bbc.com']

    def start_requests(self):
        for url in self.start_urls :
            if url == 'https://www.solidot.org':
                yield scrapy.Request(url, self.solidot_parse,meta={'url':url})

    def solidot_parse(self,response):
        for info in response.css('div.block_m'):
            a = info.css('div.talk_time::text').extract()[2]
            strinfo = re.compile(' ')
            b = strinfo.sub('',a)

            item = NewsItem()
            item['news_from'] =  response.meta['url']
            item['news_date'] = b
            item['news_title'] =  info.css('div.bg_htit h2 a::text').extract_first()
            item['news_info'] =  info.css('div.p_mainnew').extract()

            yield item

